import scrapy
import re
import math
from spiderproject.items import ZhilianItem

class Wuyou(scrapy.Spider):
    name = "wuyou"
    allowed_domains=["51job.com"]
    start_urls =["https://www.51job.com/"]
    custom_settings = {
        'ITEM_PIPELINES': {
            'spiderproject.pipelines.SpiderprojectPipeline': 1,
        },
        'CONCURRENT_REQUESTS': 50
    }

    #抓取所有分类链接
    def parse(self, response):
        class_list = response.xpath('//div[@class="cn hlist"]//a/@href').extract()
        for link in class_list:

            if  link.startswith("https://search.51job.com"):
                yield scrapy.Request(link,callback=self.all_list)
    #抓取所有列表页
    def all_list(self,response):
        base_url = self.joint_url(response.url)
        page =self.number(response.xpath('//span[@class="td"][1]//text()').extract()[0])
        for i in range(1,int(page[-1])+1):

            url = base_url.format(num=i)
            #抓取所有详情请求
            yield scrapy.Request(url,callback=self.info_urls)

    #抓取所有详情
    def info_urls(self,response):
        info_list = response.xpath('//p[@class="t1 "]//span/a/@href').extract()
        for url in info_list:
            if url.startswith("https://jobs.51job.com"):
                yield scrapy.Request(url,callback=self.parse_info)
    #解析详情
    def parse_info(self,response):
        item = ZhilianItem()
        url =  response.url

        title = response.xpath('//div[@class="cn"]//h1/text()').extract()[0]
        try:
            smoney,emoney = self.edit_gongzi(response.xpath('//div[@class="cn"]//strong/text()').extract()[0])
        except:
            smoney, emoney = 0,0
        city = response.xpath('//div[@class="cn"]//span/text()').extract()[0]
        span = response.xpath('//div[@class="t1"]//span')
        if len(span) == 3:
            release_time = self.edit_date(response.xpath('//div[@class="t1"]//span[3]/text()').extract()[0],response)
            job_type = '无'
            syear, eyear =self.year(response.xpath('//div[@class="t1"]//span[1]/text()').extract()[0])
            edu = "无"
        elif len(span) >=4:
            release_time =self.edit_date(response.xpath('//div[@class="t1"]//span[4]/text()').extract()[0],response)
            job_type = ''
            syear,eyear = self.year(response.xpath('//div[@class="t1"]//span[1]/text()').extract()[0])
            edu = response.xpath('//div[@class="t1"]//span[2]/text()').extract()[0]

        group = response.xpath('//p[@class="msg ltype"]//text()').extract()[0].split("|")[-1].strip()
        content ="".join(response.xpath('//div[@class="tBorderTop_box"][1]/div//text()').extract()).strip()
        adds ="".join(response.xpath('//p[@class="fp"][1]/text()').extract()).strip()
        label = "-".join(response.xpath('//p[@class="t2"]//span/text()').extract())

        item["url"] = url
        item["title"] = title
        item["smoney"] = smoney
        item["emoney"] = emoney
        item["city"] = city
        item["release_time"] = release_time
        item["job_type"] = job_type
        item["syear"] = syear
        item["eyear"] = eyear
        item["edu"] = edu
        item["groups"] = group
        item["content"] = content
        item["adds"] = adds
        item["label"] = label
        yield item

    # 拼接url
    def joint_url(self,value):
        list_url = value.split(',')
        list_url[-1] = '{num}.html'
        base_url = ",".join(list_url)
        return base_url

    #取数字
    def number(self,value):
        reg = re.compile(r'(\d+)')
        res = reg.findall(value)
        return res

    #发布日期
    def edit_date(self,value,response):
        if '发布'in value:
            res ='2018-'+ value.strip("发布")
        else:
            res = response.xpath('//span[@class="sp4"][3]//text()').extract()[0]
            res = '2018-'+res.strip("发布")

        return res

    #工资
    def edit_gongzi(self,value):
        if "年" in value:
            year =self. number(value)
            year = ([math.floor((int(y) * 10000) / 12) for y in year])
            smoney = year[0]
            emoney = year[1]
            return smoney,emoney
        elif '万/月' in value:
            year = value.split('-')
            smoney = int((float(year[0])) * 10000)
            emoney = int(float(year[1].strip('万/月')) * 10000)

            return smoney, emoney
        elif '千/月' in value:
            year = value.split('-')
            smoney = int((float(year[0])) * 1000)
            emoney = int(float(year[1].strip('千/月')) * 1000)

            return smoney, emoney
        elif '元/天' in value:
            year = value.strip('元/天')
            smoney = int(year) * 30
            emoney = int(year) * 30
            return smoney,emoney
        elif "千以下/月" in value:
            year = value.split('千以下/月')
            smoney = int((float(year[0]) * 1000))
            emoney =int((float(year[0])*1000))
            return smoney,emoney

    #工作年限
    def year(self,value):
        if '-' in  value:
            num =  self.number(value)
            syear = int(num[0])
            eyear = int(num[1])

        elif  "无" in value  :
            syear = 0
            eyear = 0
        else:
            num = self.number(value)
            syear = int(num[0])
            eyear = int(num[0])
        return  syear , eyear